# data = [
#     {"id": 2, "meeting": "CVPR", "time": "2024"},
#     {"id": 72, "meeting": "CVPR", "time": "2024"},
#     {"id": 71, "meeting": "CVPR", "time": "2024"},
#     {"id": 72, "meeting": "ICCV", "time": "2024"},
#     {"id": 72, "meeting": "CVPR", "time": "2025"},
#     {"id": 73, "meeting": "CVPR", "time": "2024"},
#     {"id": 72, "meeting": "CVPR", "time": ""}
# ]
import scrapy
import sys
sys.path.append('..')
# from acad_horizon_spiders.utils import jsonbd_util
from utils import basic_util,jsonbd_util
from urllib import parse

data=jsonbd_util.get_alldata("assay_detail2.json")
data=data.get('assay_detail')
print(len(data))
# 使用集合来追踪已经遇到的 ID
seen_ids = set()
unique_data = []

for item in data:
    if item["id"] not in seen_ids:
        unique_data.append(item)
        seen_ids.add(item["id"])
i=1
for item in unique_data:
    item["id"]=i
    i=i+1
jsonbd_util.update('assay_detail1.json','assay_detail',unique_data) 
print(len(unique_data))
